import java.util.Random;

public class RandomPolicy {

	final static int nepisodes = 20;
	final static int TERMINAL_STATE = -1;

    private static Random rnd = new Random( 1 );


    /**
     * First implement the equiprobable-random policy, run a number
     * episodes and observe the returns from the initial state,
     * assuming γ=1. These returns should vary between 0 and 6, with
     * an average of about 3.5. If they don’t, then you are probably
     * doing something wrong. Create this code by modifying the provided
     * file RandomPolicy.java.  Do not change the filename or class name.
     */

	public static void main( String[] arg ) {

        double avgRet = 0.0d;

		for ( int epi = 0; epi < nepisodes; epi++ ) {

            // Run an episode...
			double ret = 0.0;

            int state = Party.init();
            int numActions = Party.numActions( state );

            int action;
            int nextState;
            double reward;

			System.out.println( "Episode: " + epi );

            // do until terminal state...
            while ( state != TERMINAL_STATE ) {

                // pick an action, equiprobable-randomly from the
                // actions available from the state
                action = getRandomAction( state );

                // perform action, observe next state, reward
                nextState = Party.transition( state, action );
                reward = Party.reward( state, action, nextState );

                ret += reward;
                state = nextState;

            }

            System.out.println( "Return: " + ret );

            avgRet = avgRet + ((ret - avgRet) / ((double) epi + 1));

		}

        System.out.println( "\nAverage Return: " + avgRet + "\n" );

	}

    /**
     * Returns the equiprobable random action from a given state.
     */
    private static int getRandomAction( int state ) {

        /* Since all actions available from states where 
         * numActions(state) == 3, and where numActions(state) == 2,
         * actions 0 and 1 ( P and R ) are available and S is not,
         * we can simply return a random int between 0 and numActions - 1.
         */

        return rnd.nextInt( Party.numActions( state ) );

    }
}
